#-*- coding:utf-8 -*-
# Author: Liu Huan
# Mail: liuhuan@mail.las.ac.cn
# Datetime: 2019/7/31 16:06


import pandas as pd
import en_core_web_sm
nlp = en_core_web_sm.load()


if __name__ == '__main__':
    df = pd.read_csv('arxiv_structured.csv')
    count = 0
    with open('arxiv_structured_no_OB.txt','w',encoding='utf-8') as f:
        with open('arxiv_structured_error.txt', 'w', encoding='utf-8') as fw:
            for i in range(len(df)):
                id_ = df.iloc[i]['ID']
                abs = df.iloc[i]['abstracts']
                abs = abs.lower()
                if abs.startswith('aims.'):
                    f_str = ''
                    try:
                        # OBJECTIVE
                        segs = abs.split('methods.')
                        move = segs[0][5:].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'OBJECTIVE' + '\t' + sen +'\n'
                        # METHODS
                        segs = segs[1].split('results.')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'METHODS' + '\t' + sen +'\n'
                        # RESULTS
                        segs = segs[1].split('conclusions.')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'RESULTS' + '\t' + sen + '\n'
                        # CONCLUSIONS
                        move = segs[1].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'CONCLUSIONS' + '\t' + sen + '\n'
                        f.write('###' + str(id_) + '\n')
                        f.write(f_str)
                        f.write('\n')
                        count += 1
                    except:
                        fw.write('###' + str(id_) + '\n')
                        fw.write(abs+ '\n' )
                elif abs.startswith('context.') and 'aims.' not in abs:
                    f_str = ''
                    try:
                        # BACKGROUND
                        segs = abs.split('methods.')
                        move = segs[0][8:].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'BACKGROUND' + '\t' + sen + '\n'
                        # METHODS
                        segs = segs[1].split('results.')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'METHODS' + '\t' + sen + '\n'
                        # RESULTS
                        segs = segs[1].split('conclusions.')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'RESULTS' + '\t' + sen + '\n'
                        # CONCLUSIONS
                        move = segs[1].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'CONCLUSIONS' + '\t' + sen + '\n'
                        f.write('###' + str(id_) + '\n')
                        f.write(f_str)
                        f.write('\n')
                        count += 1
                    except:
                        fw.write('###' + str(id_) + '\n')
                        fw.write(abs + '\n')
                elif abs.startswith('background.') and 'aims.' not in abs:
                    f_str = ''
                    try:
                        # BACKGROUND
                        segs = abs.split('methods.')
                        move = segs[0][11:].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'BACKGROUND' + '\t' + sen + '\n'
                        # METHODS
                        segs = segs[1].split('results.')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'METHODS' + '\t' + sen + '\n'
                        # RESULTS
                        segs = segs[1].split('conclusions.')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'RESULTS' + '\t' + sen + '\n'
                        # CONCLUSIONS
                        move = segs[1].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'CONCLUSIONS' + '\t' + sen + '\n'
                        f.write('###' + str(id_) + '\n')
                        f.write(f_str)
                        f.write('\n')
                        count += 1
                    except:
                        fw.write('###' + str(id_) + '\n')
                        fw.write(abs + '\n')
                elif abs.startswith('aims:'):
                    f_str = ''
                    try:
                        # OBJECTIVE
                        segs = abs.split('methods:')
                        move = segs[0][5:].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'OBJECTIVE' + '\t' + sen +'\n'
                        # METHODS
                        segs = segs[1].split('results:')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'METHODS' + '\t' + sen +'\n'
                        # RESULTS
                        segs = segs[1].split('conclusions:')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'RESULTS' + '\t' + sen + '\n'
                        # CONCLUSIONS
                        move = segs[1].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'CONCLUSIONS' + '\t' + sen + '\n'
                        f.write('###' + str(id_) + '\n')
                        f.write(f_str)
                        f.write('\n')
                        count += 1
                    except:
                        fw.write('###' + str(id_) + '\n')
                        fw.write(abs+ '\n' )
                elif abs.startswith('context:') and 'aims:' not in abs:
                    f_str = ''
                    try:
                        # BACKGROUND
                        segs = abs.split('methods:')
                        move = segs[0][8:].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'BACKGROUND' + '\t' + sen + '\n'
                        # METHODS
                        segs = segs[1].split('results:')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'METHODS' + '\t' + sen + '\n'
                        # RESULTS
                        segs = segs[1].split('conclusions:')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'RESULTS' + '\t' + sen + '\n'
                        # CONCLUSIONS
                        move = segs[1].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'CONCLUSIONS' + '\t' + sen + '\n'
                        f.write('###' + str(id_) + '\n')
                        f.write(f_str)
                        f.write('\n')
                        count += 1
                    except:
                        fw.write('###' + str(id_) + '\n')
                        fw.write(abs + '\n')
                elif abs.startswith('context :') and 'aims :' not in abs:
                    f_str = ''
                    try:
                        # BACKGROUND
                        segs = abs.split('methods :')
                        move = segs[0][9:].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'BACKGROUND' + '\t' + sen + '\n'
                        # METHODS
                        segs = segs[1].split('results :')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'METHODS' + '\t' + sen + '\n'
                        # RESULTS
                        segs = segs[1].split('conclusions :')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'RESULTS' + '\t' + sen + '\n'
                        # CONCLUSIONS
                        move = segs[1].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'CONCLUSIONS' + '\t' + sen + '\n'
                        f.write('###' + str(id_) + '\n')
                        f.write(f_str)
                        f.write('\n')
                        count += 1
                    except:
                        fw.write('###' + str(id_) + '\n')
                        fw.write(abs + '\n')
                elif abs.startswith('background:') and 'aims:' not in abs:
                    f_str = ''
                    try:
                        # BACKGROUND
                        segs = abs.split('methods:')
                        move = segs[0][11:].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'BACKGROUND' + '\t' + sen + '\n'
                        # METHODS
                        segs = segs[1].split('results:')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'METHODS' + '\t' + sen + '\n'
                        # RESULTS
                        segs = segs[1].split('conclusions:')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'RESULTS' + '\t' + sen + '\n'
                        # CONCLUSIONS
                        move = segs[1].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'CONCLUSIONS' + '\t' + sen + '\n'
                        f.write('###' + str(id_) + '\n')
                        f.write(f_str)
                        f.write('\n')
                        count += 1
                    except:
                        fw.write('###' + str(id_) + '\n')
                        fw.write(abs + '\n')
                elif abs.startswith('background :') and 'aims :' not in abs:
                    f_str = ''
                    try:
                        # BACKGROUND
                        segs = abs.split('methods :')
                        move = segs[0][12:].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'BACKGROUND' + '\t' + sen + '\n'
                        # METHODS
                        segs = segs[1].split('results :')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'METHODS' + '\t' + sen + '\n'
                        # RESULTS
                        segs = segs[1].split('conclusions :')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'RESULTS' + '\t' + sen + '\n'
                        # CONCLUSIONS
                        move = segs[1].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'CONCLUSIONS' + '\t' + sen + '\n'
                        f.write('###' + str(id_) + '\n')
                        f.write(f_str)
                        f.write('\n')
                        count += 1
                    except:
                        fw.write('###' + str(id_) + '\n')
                        fw.write(abs + '\n')
                elif abs.startswith('aims :'):
                    f_str = ''
                    try:
                        # OBJECTIVE
                        segs = abs.split('methods :')
                        move = segs[0][6:].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'OBJECTIVE' + '\t' + sen +'\n'
                        # METHODS
                        segs = segs[1].split('results :')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'METHODS' + '\t' + sen +'\n'
                        # RESULTS
                        segs = segs[1].split('conclusions :')
                        move = segs[0].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'RESULTS' + '\t' + sen + '\n'
                        # CONCLUSIONS
                        move = segs[1].strip()
                        for sen in nlp(move).sents:
                            sen = str(sen)
                            f_str += 'CONCLUSIONS' + '\t' + sen + '\n'
                        f.write('###' + str(id_) + '\n')
                        f.write(f_str)
                        f.write('\n')
                        count += 1
                    except:
                        fw.write('###' + str(id_) + '\n')
                        fw.write(abs+ '\n' )
                # elif abs.startswith('context {'):
                #     count += 1
                # elif abs.startswith('context -'):
                #     count += 1
                # elif abs.startswith('aims -'):
                #     count += 1
                # elif abs.startswith('context -'):
                #     count += 1
                # elif abs.startswith('background-'):
                #     count += 1
                # # elif abs.startswith('aims '):
                # #     count += 1
                # # elif abs.startswith('context '):
                # #     co unt += 1
                # elif abs.startswith('background/introduction:'):
                #     count += 1
                # elif abs.startswith('contextuality') or abs.startswith('background quasars ') or abs.startswith('background events') or abs.startswith('background properties'):
                #     pass
                else:
                    fw.write('###' + str(id_) + '\n')
                    fw.write(abs + '\n')
                print(count)
    # print(count)
    print(i+1-count)